{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "strange-cartoon",
   "metadata": {},
   "source": [
    "# TimeEval shared parameter optimization result analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "elegant-fellow",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Automatically reload packages:\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "directed-instruction",
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import json\n",
    "import warnings\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy as sp\n",
    "import plotly.offline as py\n",
    "import plotly.graph_objects as go\n",
    "import plotly.figure_factory as ff\n",
    "import plotly.express as px\n",
    "from plotly.subplots import make_subplots\n",
    "from pathlib import Path\n",
    "from timeeval import Datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "synthetic-motivation",
   "metadata": {},
   "source": [
    "## Configuration\n",
    "\n",
    "Target parameters that were optimized in this run (per algorithm):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4d114231",
   "metadata": {},
   "outputs": [],
   "source": [
    "algo_param_mapping = {\n",
    "  \"HBOS\": [\"n_bins\"],\n",
    "  \"MultiHMM\": [\"n_bins\"],\n",
    "  \"MTAD-GAT\": [\"context_window_size\", \"mag_window_size\", \"score_window_size\"],\n",
    "  \"PST\": [\"n_bins\"]\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "surface-tanzania",
   "metadata": {},
   "source": [
    "Define data and results folder:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "textile-preview",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available result directories:\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[PosixPath('../timeeval_experiments/results/2021-09-30-torsk'),\n",
       " PosixPath('../timeeval_experiments/results/2021-09-27_shared-optim'),\n",
       " PosixPath('../timeeval_experiments/results/2021-10-04_shared-optim2')]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Selecting:\n",
      "Data path: /home/sebastian/Documents/Projects/akita/data/test-cases\n",
      "Result path: /home/sebastian/Documents/Projects/akita/timeeval/timeeval_experiments/results/2021-10-04_shared-optim2\n"
     ]
    }
   ],
   "source": [
    "# constants and configuration\n",
    "data_path = Path(\"../../data\") / \"test-cases\"\n",
    "result_root_path = Path(\"../timeeval_experiments/results\")\n",
    "experiment_result_folder = \"2021-10-04_shared-optim2\"\n",
    "\n",
    "# build paths\n",
    "result_paths = [d for d in result_root_path.iterdir() if d.is_dir()]\n",
    "print(\"Available result directories:\")\n",
    "display(result_paths)\n",
    "\n",
    "result_path = result_root_path / experiment_result_folder\n",
    "print(\"\\nSelecting:\")\n",
    "print(f\"Data path: {data_path.resolve()}\")\n",
    "print(f\"Result path: {result_path.resolve()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a158ae1",
   "metadata": {},
   "source": [
    "Load results and dataset metadata:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "02af6424",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading results from /home/sebastian/Documents/Projects/akita/timeeval/timeeval_experiments/results/2021-10-04_shared-optim2\n"
     ]
    }
   ],
   "source": [
    "def extract_hyper_params(param_names):\n",
    "    def extract(value):\n",
    "        params = json.loads(value)\n",
    "        result = None\n",
    "        for name in param_names:\n",
    "            try:\n",
    "                value = params[name]\n",
    "                result = pd.Series([name, value], index=[\"optim_param_name\", \"optim_param_value\"])\n",
    "                break\n",
    "            except KeyError:\n",
    "                pass\n",
    "        if result is None:\n",
    "            raise ValueError(f\"Parameters {param_names} not found in '{value}'\")\n",
    "        return result\n",
    "    return extract\n",
    "\n",
    "# load results\n",
    "print(f\"Reading results from {result_path.resolve()}\")\n",
    "df = pd.read_csv(result_path / \"results.csv\")\n",
    "\n",
    "# add dataset_name column\n",
    "df[\"dataset_name\"] = df[\"dataset\"].str.split(\".\").str[0]\n",
    "\n",
    "# add optim_params column\n",
    "df[[\"optim_param_name\", \"optim_param_value\"]] = \"\"\n",
    "for algo in algo_param_mapping:\n",
    "    df_algo = df.loc[df[\"algorithm\"] == algo]\n",
    "    df.loc[df_algo.index, [\"optim_param_name\", \"optim_param_value\"]] = df_algo[\"hyper_params\"].apply(extract_hyper_params(algo_param_mapping[algo]))\n",
    "\n",
    "# load dataset metadata\n",
    "dmgr = Datasets(data_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "72dfe34f",
   "metadata": {},
   "source": [
    "Define plotting functions:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "spiritual-emergency",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_scores_df(algorithm_name, dataset_id, optim_params, repetition=1):\n",
    "    params_id = df.loc[(df[\"algorithm\"] == algorithm_name) & (df[\"collection\"] == dataset_id[0]) & (df[\"dataset\"] == dataset_id[1]) & (df[\"optim_param_name\"] == optim_params[0]) & (df[\"optim_param_value\"] == optim_params[1]), \"hyper_params_id\"].item()\n",
    "    path = (\n",
    "        result_path /\n",
    "        algorithm_name /\n",
    "        params_id /\n",
    "        dataset_id[0] /\n",
    "        dataset_id[1] /\n",
    "        str(repetition) /\n",
    "        \"anomaly_scores.ts\"\n",
    "    )\n",
    "    return pd.read_csv(path, header=None)\n",
    "\n",
    "def plot_scores(algorithm_name, dataset_name):\n",
    "    if isinstance(algorithm_name, tuple):\n",
    "        algorithms = [algorithm_name]\n",
    "    elif not isinstance(algorithm_name, list):\n",
    "        raise ValueError(\"Please supply a tuple (algorithm_name, optim_param_name, optim_param_value) or a list thereof as first argument!\")\n",
    "    else:\n",
    "        algorithms = algorithm_name\n",
    "    # construct dataset ID\n",
    "    dataset_id = (\"GutenTAG\", f\"{dataset_name}.unsupervised\")\n",
    "\n",
    "    # load dataset details\n",
    "    df_dataset = dmgr.get_dataset_df(dataset_id)\n",
    "\n",
    "    # check if dataset is multivariate\n",
    "    dataset_dim = df.loc[df[\"dataset_name\"] == dataset_name, \"dataset_input_dimensionality\"].unique().item()\n",
    "    dataset_dim = dataset_dim.lower()\n",
    "    \n",
    "    auroc = {}\n",
    "    df_scores = pd.DataFrame(index=df_dataset.index)\n",
    "    skip_algos = []\n",
    "    algos = []\n",
    "    for algo, optim_param_name, optim_param_value in algorithms:\n",
    "        optim_params = f\"{optim_param_name}={optim_param_value}\"\n",
    "        algos.append((algo, optim_params))\n",
    "        # get algorithm metric results\n",
    "        try:\n",
    "            auroc[(algo, optim_params)] = df.loc[\n",
    "                (df[\"algorithm\"] == algo) & (df[\"dataset_name\"] == dataset_name) & (df[\"optim_param_name\"] == optim_param_name) & (df[\"optim_param_value\"] == optim_param_value),\n",
    "                \"ROC_AUC\"\n",
    "            ].item()\n",
    "        except ValueError:\n",
    "            warnings.warn(f\"No ROC_AUC score found! Probably {algo} with params {optim_params} was not executed on {dataset_name}.\")\n",
    "            auroc[(algo, optim_params)] = -1\n",
    "            skip_algos.append((algo, optim_params))\n",
    "            continue\n",
    "\n",
    "        # load scores\n",
    "        training_type = df.loc[df[\"algorithm\"] == algo, \"algo_training_type\"].values[0].lower().replace(\"_\", \"-\")\n",
    "        try:\n",
    "            df_scores[(algo, optim_params)] = load_scores_df(algo, (\"GutenTAG\", f\"{dataset_name}.{training_type}\"), (optim_param_name, optim_param_value)).iloc[:, 0]\n",
    "        except (ValueError, FileNotFoundError):\n",
    "            warnings.warn(f\"No anomaly scores found! Probably {algo} was not executed on {dataset_name} with params {optim_params}.\")\n",
    "            df_scores[(algo, optim_params)] = np.nan\n",
    "            skip_algos.append((algo, optim_params))\n",
    "    algorithms = [a for a in algos if a not in skip_algos]\n",
    "\n",
    "    # Create plot\n",
    "    fig = make_subplots(2, 1)\n",
    "    if dataset_dim == \"multivariate\":\n",
    "        for i in range(1, df_dataset.shape[1]-1):\n",
    "            fig.add_trace(go.Scatter(x=df_dataset.index, y=df_dataset.iloc[:, i], name=f\"channel-{i}\"), 1, 1)\n",
    "    else:\n",
    "        fig.add_trace(go.Scatter(x=df_dataset.index, y=df_dataset.iloc[:, 1], name=\"timeseries\"), 1, 1)\n",
    "    fig.add_trace(go.Scatter(x=df_dataset.index, y=df_dataset[\"is_anomaly\"], name=\"label\"), 2, 1)\n",
    "    \n",
    "    for item in algorithms:\n",
    "        algo, optim_params = item\n",
    "        fig.add_trace(go.Scatter(x=df_scores.index, y=df_scores[item], name=f\"{algo}={auroc[item]:.4f} ({optim_params})\"), 2, 1)\n",
    "    fig.update_xaxes(matches=\"x\")\n",
    "    fig.update_layout(\n",
    "        title=f\"Results of {','.join(np.unique([a for a, _ in algorithms]))} on {dataset_name}\",\n",
    "        height=400\n",
    "    )\n",
    "    return py.iplot(fig)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "julian-produce",
   "metadata": {},
   "source": [
    "## Analyze TimeEval results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "disturbed-impact",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>algorithm</th>\n",
       "      <th>dataset_name</th>\n",
       "      <th>status</th>\n",
       "      <th>AVERAGE_PRECISION</th>\n",
       "      <th>PR_AUC</th>\n",
       "      <th>RANGE_PR_AUC</th>\n",
       "      <th>ROC_AUC</th>\n",
       "      <th>execute_main_time</th>\n",
       "      <th>optim_param_name</th>\n",
       "      <th>optim_param_value</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HBOS</td>\n",
       "      <td>ecg-channels-all-of-3</td>\n",
       "      <td>Status.OK</td>\n",
       "      <td>0.032220</td>\n",
       "      <td>0.150289</td>\n",
       "      <td>0.143813</td>\n",
       "      <td>0.544104</td>\n",
       "      <td>26.899593</td>\n",
       "      <td>n_bins</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HBOS</td>\n",
       "      <td>ecg-channels-single-of-10</td>\n",
       "      <td>Status.OK</td>\n",
       "      <td>0.012512</td>\n",
       "      <td>0.012796</td>\n",
       "      <td>0.010695</td>\n",
       "      <td>0.577128</td>\n",
       "      <td>31.133166</td>\n",
       "      <td>n_bins</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HBOS</td>\n",
       "      <td>ecg-channels-single-of-2</td>\n",
       "      <td>Status.OK</td>\n",
       "      <td>0.010035</td>\n",
       "      <td>0.010804</td>\n",
       "      <td>0.012908</td>\n",
       "      <td>0.530409</td>\n",
       "      <td>33.759874</td>\n",
       "      <td>n_bins</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HBOS</td>\n",
       "      <td>ecg-channels-single-of-20</td>\n",
       "      <td>Status.OK</td>\n",
       "      <td>0.012632</td>\n",
       "      <td>0.012819</td>\n",
       "      <td>0.011020</td>\n",
       "      <td>0.577631</td>\n",
       "      <td>33.053460</td>\n",
       "      <td>n_bins</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HBOS</td>\n",
       "      <td>ecg-channels-single-of-5</td>\n",
       "      <td>Status.OK</td>\n",
       "      <td>0.010447</td>\n",
       "      <td>0.011373</td>\n",
       "      <td>0.010637</td>\n",
       "      <td>0.547163</td>\n",
       "      <td>29.512747</td>\n",
       "      <td>n_bins</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3480</th>\n",
       "      <td>PST</td>\n",
       "      <td>sinus-type-pattern-shift</td>\n",
       "      <td>Status.OK</td>\n",
       "      <td>0.014713</td>\n",
       "      <td>0.014132</td>\n",
       "      <td>0.011032</td>\n",
       "      <td>0.472318</td>\n",
       "      <td>26.676900</td>\n",
       "      <td>n_bins</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3481</th>\n",
       "      <td>PST</td>\n",
       "      <td>sinus-type-pattern</td>\n",
       "      <td>Status.OK</td>\n",
       "      <td>0.005067</td>\n",
       "      <td>0.005017</td>\n",
       "      <td>0.028875</td>\n",
       "      <td>0.001809</td>\n",
       "      <td>23.909352</td>\n",
       "      <td>n_bins</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3482</th>\n",
       "      <td>PST</td>\n",
       "      <td>sinus-type-platform</td>\n",
       "      <td>Status.OK</td>\n",
       "      <td>0.005106</td>\n",
       "      <td>0.005054</td>\n",
       "      <td>0.005189</td>\n",
       "      <td>0.034916</td>\n",
       "      <td>23.341548</td>\n",
       "      <td>n_bins</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3483</th>\n",
       "      <td>PST</td>\n",
       "      <td>sinus-type-trend</td>\n",
       "      <td>Status.OK</td>\n",
       "      <td>0.011853</td>\n",
       "      <td>0.011658</td>\n",
       "      <td>0.015139</td>\n",
       "      <td>0.577974</td>\n",
       "      <td>22.664949</td>\n",
       "      <td>n_bins</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3484</th>\n",
       "      <td>PST</td>\n",
       "      <td>sinus-type-variance</td>\n",
       "      <td>Status.OK</td>\n",
       "      <td>0.005067</td>\n",
       "      <td>0.005017</td>\n",
       "      <td>0.180609</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>23.546426</td>\n",
       "      <td>n_bins</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3485 rows × 10 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     algorithm               dataset_name     status  AVERAGE_PRECISION  \\\n",
       "0         HBOS      ecg-channels-all-of-3  Status.OK           0.032220   \n",
       "1         HBOS  ecg-channels-single-of-10  Status.OK           0.012512   \n",
       "2         HBOS   ecg-channels-single-of-2  Status.OK           0.010035   \n",
       "3         HBOS  ecg-channels-single-of-20  Status.OK           0.012632   \n",
       "4         HBOS   ecg-channels-single-of-5  Status.OK           0.010447   \n",
       "...        ...                        ...        ...                ...   \n",
       "3480       PST   sinus-type-pattern-shift  Status.OK           0.014713   \n",
       "3481       PST         sinus-type-pattern  Status.OK           0.005067   \n",
       "3482       PST        sinus-type-platform  Status.OK           0.005106   \n",
       "3483       PST           sinus-type-trend  Status.OK           0.011853   \n",
       "3484       PST        sinus-type-variance  Status.OK           0.005067   \n",
       "\n",
       "        PR_AUC  RANGE_PR_AUC   ROC_AUC  execute_main_time optim_param_name  \\\n",
       "0     0.150289      0.143813  0.544104          26.899593           n_bins   \n",
       "1     0.012796      0.010695  0.577128          31.133166           n_bins   \n",
       "2     0.010804      0.012908  0.530409          33.759874           n_bins   \n",
       "3     0.012819      0.011020  0.577631          33.053460           n_bins   \n",
       "4     0.011373      0.010637  0.547163          29.512747           n_bins   \n",
       "...        ...           ...       ...                ...              ...   \n",
       "3480  0.014132      0.011032  0.472318          26.676900           n_bins   \n",
       "3481  0.005017      0.028875  0.001809          23.909352           n_bins   \n",
       "3482  0.005054      0.005189  0.034916          23.341548           n_bins   \n",
       "3483  0.011658      0.015139  0.577974          22.664949           n_bins   \n",
       "3484  0.005017      0.180609  0.000001          23.546426           n_bins   \n",
       "\n",
       "     optim_param_value  \n",
       "0                    5  \n",
       "1                    5  \n",
       "2                    5  \n",
       "3                    5  \n",
       "4                    5  \n",
       "...                ...  \n",
       "3480                20  \n",
       "3481                20  \n",
       "3482                20  \n",
       "3483                20  \n",
       "3484                20  \n",
       "\n",
       "[3485 rows x 10 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[[\"algorithm\", \"dataset_name\", \"status\", \"AVERAGE_PRECISION\", \"PR_AUC\", \"RANGE_PR_AUC\", \"ROC_AUC\", \"execute_main_time\", \"optim_param_name\", \"optim_param_value\"]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "casual-rubber",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "incident-clarity",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "df_error_counts = df.pivot_table(index=[\"algo_training_type\", \"algorithm\"], columns=[\"status\"], values=\"repetition\", aggfunc=\"count\")\n",
    "df_error_counts = df_error_counts.fillna(value=0).astype(np.int64)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "signed-recycling",
   "metadata": {},
   "source": [
    "#### Aggregation of errors per algorithm grouped by algorithm training type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "sharing-lewis",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SEMI_SUPERVISED\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>status</th>\n",
       "      <th>Status.ERROR</th>\n",
       "      <th>Status.OK</th>\n",
       "      <th>Status.TIMEOUT</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>algorithm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MTAD-GAT</th>\n",
       "      <td>180</td>\n",
       "      <td>480</td>\n",
       "      <td>1045</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "status     Status.ERROR  Status.OK  Status.TIMEOUT\n",
       "algorithm                                         \n",
       "MTAD-GAT            180        480            1045"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SUPERVISED\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>status</th>\n",
       "      <th>Status.ERROR</th>\n",
       "      <th>Status.OK</th>\n",
       "      <th>Status.TIMEOUT</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>algorithm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MultiHMM</th>\n",
       "      <td>125</td>\n",
       "      <td>494</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "status     Status.ERROR  Status.OK  Status.TIMEOUT\n",
       "algorithm                                         \n",
       "MultiHMM            125        494               1"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "UNSUPERVISED\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>status</th>\n",
       "      <th>Status.ERROR</th>\n",
       "      <th>Status.OK</th>\n",
       "      <th>Status.TIMEOUT</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>algorithm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>HBOS</th>\n",
       "      <td>0</td>\n",
       "      <td>620</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>PST</th>\n",
       "      <td>0</td>\n",
       "      <td>540</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "status     Status.ERROR  Status.OK  Status.TIMEOUT\n",
       "algorithm                                         \n",
       "HBOS                  0        620               0\n",
       "PST                   0        540               0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for tpe in [\"SEMI_SUPERVISED\", \"SUPERVISED\", \"UNSUPERVISED\"]:\n",
    "    if tpe in df_error_counts.index:\n",
    "        print(tpe)\n",
    "        display(df_error_counts.loc[tpe])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "quantitative-wednesday",
   "metadata": {},
   "source": [
    "#### Slow algorithms\n",
    "\n",
    "Algorithms, for which more than 50% of all executions ran into the timeout."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bigger-africa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>status</th>\n",
       "      <th>Status.ERROR</th>\n",
       "      <th>Status.OK</th>\n",
       "      <th>Status.TIMEOUT</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>algo_training_type</th>\n",
       "      <th>algorithm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>SEMI_SUPERVISED</th>\n",
       "      <th>MTAD-GAT</th>\n",
       "      <td>180</td>\n",
       "      <td>480</td>\n",
       "      <td>1045</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "status                        Status.ERROR  Status.OK  Status.TIMEOUT\n",
       "algo_training_type algorithm                                         \n",
       "SEMI_SUPERVISED    MTAD-GAT            180        480            1045"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_error_counts[df_error_counts[\"Status.TIMEOUT\"] > (df_error_counts[\"Status.ERROR\"] + df_error_counts[\"Status.OK\"])]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "shaped-outside",
   "metadata": {},
   "source": [
    "#### Broken algorithms\n",
    "\n",
    "Algorithms, which failed for at least 50% of the executions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "optical-elder",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>status</th>\n",
       "      <th>Status.ERROR</th>\n",
       "      <th>Status.OK</th>\n",
       "      <th>Status.TIMEOUT</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>algo_training_type</th>\n",
       "      <th>algorithm</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [Status.ERROR, Status.OK, Status.TIMEOUT]\n",
       "Index: []"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "error_threshold = 0.5\n",
    "df_error_counts[df_error_counts[\"Status.ERROR\"] > error_threshold*(\n",
    "    df_error_counts[\"Status.TIMEOUT\"] + df_error_counts[\"Status.ERROR\"] + df_error_counts[\"Status.OK\"]\n",
    ")]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97010441",
   "metadata": {},
   "source": [
    "#### Detail errors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "68b1af05",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>OOM</th>\n",
       "      <th>Segfault</th>\n",
       "      <th>ZeroDivisionError</th>\n",
       "      <th>IncompatibleParameterConfig</th>\n",
       "      <th>WrongDBNState</th>\n",
       "      <th>SyntaxError</th>\n",
       "      <th>other</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MTAD-GAT</th>\n",
       "      <td>180</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MultiHMM</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>125</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          OOM  Segfault  ZeroDivisionError  IncompatibleParameterConfig  \\\n",
       "MTAD-GAT  180         0                  0                            0   \n",
       "MultiHMM    0         0                  0                            0   \n",
       "\n",
       "          WrongDBNState  SyntaxError  other  \n",
       "MTAD-GAT              0            0      0  \n",
       "MultiHMM              0          125      0  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "algo_list = [\"MTAD-GAT\", \"MultiHMM\"]\n",
    "\n",
    "error_list = [\"OOM\", \"Segfault\", \"ZeroDivisionError\", \"IncompatibleParameterConfig\", \"WrongDBNState\", \"SyntaxError\", \"other\"]\n",
    "errors = pd.DataFrame(0, index=error_list, columns=algo_list, dtype=np.int_)\n",
    "for algo in algo_list:\n",
    "    df_tmp = df[(df[\"algorithm\"] == algo) & (df[\"status\"] == \"Status.ERROR\")]\n",
    "    for i, run in df_tmp.iterrows():\n",
    "        path = result_path / run[\"algorithm\"] / run[\"hyper_params_id\"] / run[\"collection\"] / run[\"dataset\"] / str(run[\"repetition\"]) / \"execution.log\"\n",
    "        with path.open(\"r\") as fh:\n",
    "            log = fh.read()\n",
    "            if \"status code '139'\" in log:\n",
    "                errors.loc[\"Segfault\", algo] += 1\n",
    "            elif \"status code '137'\" in log:\n",
    "                errors.loc[\"OOM\", algo] += 1\n",
    "            elif \"Expected n_neighbors <= n_samples\" in log:\n",
    "                errors.loc[\"IncompatibleParameterConfig\", algo] += 1\n",
    "            elif \"ZeroDivisionError\" in log:\n",
    "                errors.loc[\"ZeroDivisionError\", algo] += 1\n",
    "            elif \"does not have key\" in log:\n",
    "                errors.loc[\"WrongDBNState\", algo] += 1\n",
    "            elif \"NameError\" in log:\n",
    "                errors.loc[\"SyntaxError\", algo] += 1\n",
    "            else:\n",
    "                print(f'\\n\\n#### {run[\"dataset\"]} ({run[\"optim_param_name\"]}:{run[\"optim_param_value\"]})')\n",
    "                print(log)\n",
    "                errors.loc[\"other\", algo] += 1\n",
    "errors.T"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "educated-observer",
   "metadata": {},
   "source": [
    "---\n",
    "\n",
    "### Parameter assessment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "32ebedf7",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th colspan=\"2\" halign=\"left\">AVERAGE_PRECISION</th>\n",
       "      <th colspan=\"2\" halign=\"left\">RANGE_PR_AUC</th>\n",
       "      <th colspan=\"2\" halign=\"left\">PR_AUC</th>\n",
       "      <th colspan=\"2\" halign=\"left\">ROC_AUC</th>\n",
       "      <th>train_main_time</th>\n",
       "      <th>execute_main_time</th>\n",
       "      <th>repetition</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>mean</th>\n",
       "      <th>median</th>\n",
       "      <th>mean</th>\n",
       "      <th>mean</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>algorithm</th>\n",
       "      <th>optim_param_name</th>\n",
       "      <th>optim_param_value</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">PST</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">n_bins</th>\n",
       "      <th>5</th>\n",
       "      <td>0.397080</td>\n",
       "      <td>0.317714</td>\n",
       "      <td>0.385879</td>\n",
       "      <td>0.302098</td>\n",
       "      <td>0.396243</td>\n",
       "      <td>0.317273</td>\n",
       "      <td>0.802249</td>\n",
       "      <td>0.894860</td>\n",
       "      <td>NaN</td>\n",
       "      <td>34.725921</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.272842</td>\n",
       "      <td>0.076066</td>\n",
       "      <td>0.268959</td>\n",
       "      <td>0.120117</td>\n",
       "      <td>0.271491</td>\n",
       "      <td>0.074917</td>\n",
       "      <td>0.698583</td>\n",
       "      <td>0.794044</td>\n",
       "      <td>NaN</td>\n",
       "      <td>35.959607</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.202958</td>\n",
       "      <td>0.035208</td>\n",
       "      <td>0.220957</td>\n",
       "      <td>0.101009</td>\n",
       "      <td>0.202411</td>\n",
       "      <td>0.034813</td>\n",
       "      <td>0.592948</td>\n",
       "      <td>0.689568</td>\n",
       "      <td>NaN</td>\n",
       "      <td>37.905403</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.072902</td>\n",
       "      <td>0.014033</td>\n",
       "      <td>0.119969</td>\n",
       "      <td>0.064917</td>\n",
       "      <td>0.072639</td>\n",
       "      <td>0.013889</td>\n",
       "      <td>0.399095</td>\n",
       "      <td>0.298680</td>\n",
       "      <td>NaN</td>\n",
       "      <td>46.914917</td>\n",
       "      <td>135</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">MultiHMM</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">n_bins</th>\n",
       "      <th>8</th>\n",
       "      <td>0.039097</td>\n",
       "      <td>0.014123</td>\n",
       "      <td>0.133103</td>\n",
       "      <td>0.026042</td>\n",
       "      <td>0.038271</td>\n",
       "      <td>0.009875</td>\n",
       "      <td>0.476832</td>\n",
       "      <td>0.471447</td>\n",
       "      <td>363.003773</td>\n",
       "      <td>6.446919</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.069705</td>\n",
       "      <td>0.016897</td>\n",
       "      <td>0.130878</td>\n",
       "      <td>0.022867</td>\n",
       "      <td>0.070247</td>\n",
       "      <td>0.011805</td>\n",
       "      <td>0.470824</td>\n",
       "      <td>0.480316</td>\n",
       "      <td>97.734562</td>\n",
       "      <td>6.673174</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.031629</td>\n",
       "      <td>0.013809</td>\n",
       "      <td>0.143500</td>\n",
       "      <td>0.027247</td>\n",
       "      <td>0.033808</td>\n",
       "      <td>0.009214</td>\n",
       "      <td>0.461211</td>\n",
       "      <td>0.460252</td>\n",
       "      <td>342.761122</td>\n",
       "      <td>6.536975</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.031147</td>\n",
       "      <td>0.011715</td>\n",
       "      <td>0.156079</td>\n",
       "      <td>0.039948</td>\n",
       "      <td>0.029807</td>\n",
       "      <td>0.009737</td>\n",
       "      <td>0.452648</td>\n",
       "      <td>0.467494</td>\n",
       "      <td>427.667099</td>\n",
       "      <td>6.671899</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"11\" valign=\"top\">MTAD-GAT</th>\n",
       "      <th rowspan=\"3\" valign=\"top\">score_window_size</th>\n",
       "      <th>52</th>\n",
       "      <td>0.208309</td>\n",
       "      <td>0.094298</td>\n",
       "      <td>0.219297</td>\n",
       "      <td>0.137429</td>\n",
       "      <td>0.206107</td>\n",
       "      <td>0.092904</td>\n",
       "      <td>0.650048</td>\n",
       "      <td>0.653008</td>\n",
       "      <td>7211.930384</td>\n",
       "      <td>503.538815</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.208844</td>\n",
       "      <td>0.091680</td>\n",
       "      <td>0.225610</td>\n",
       "      <td>0.151763</td>\n",
       "      <td>0.207560</td>\n",
       "      <td>0.089998</td>\n",
       "      <td>0.612438</td>\n",
       "      <td>0.627799</td>\n",
       "      <td>7211.671927</td>\n",
       "      <td>564.458770</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.195383</td>\n",
       "      <td>0.074781</td>\n",
       "      <td>0.210971</td>\n",
       "      <td>0.117486</td>\n",
       "      <td>0.193867</td>\n",
       "      <td>0.073706</td>\n",
       "      <td>0.606485</td>\n",
       "      <td>0.579363</td>\n",
       "      <td>7211.709492</td>\n",
       "      <td>525.772179</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">mag_window_size</th>\n",
       "      <th>40</th>\n",
       "      <td>0.231323</td>\n",
       "      <td>0.096916</td>\n",
       "      <td>0.222969</td>\n",
       "      <td>0.110509</td>\n",
       "      <td>0.228938</td>\n",
       "      <td>0.096297</td>\n",
       "      <td>0.668495</td>\n",
       "      <td>0.685376</td>\n",
       "      <td>7211.614644</td>\n",
       "      <td>525.751523</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.216599</td>\n",
       "      <td>0.094298</td>\n",
       "      <td>0.224883</td>\n",
       "      <td>0.143278</td>\n",
       "      <td>0.214355</td>\n",
       "      <td>0.092904</td>\n",
       "      <td>0.627630</td>\n",
       "      <td>0.603581</td>\n",
       "      <td>7211.677360</td>\n",
       "      <td>566.705334</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>0.236469</td>\n",
       "      <td>0.097493</td>\n",
       "      <td>0.244833</td>\n",
       "      <td>0.168187</td>\n",
       "      <td>0.235039</td>\n",
       "      <td>0.096312</td>\n",
       "      <td>0.616610</td>\n",
       "      <td>0.571274</td>\n",
       "      <td>7211.752869</td>\n",
       "      <td>513.039253</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">context_window_size</th>\n",
       "      <th>30</th>\n",
       "      <td>0.260921</td>\n",
       "      <td>0.097392</td>\n",
       "      <td>0.254209</td>\n",
       "      <td>0.142827</td>\n",
       "      <td>0.259156</td>\n",
       "      <td>0.096297</td>\n",
       "      <td>0.666837</td>\n",
       "      <td>0.698582</td>\n",
       "      <td>7211.686857</td>\n",
       "      <td>515.276039</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.198362</td>\n",
       "      <td>0.097154</td>\n",
       "      <td>0.214367</td>\n",
       "      <td>0.161455</td>\n",
       "      <td>0.196483</td>\n",
       "      <td>0.096053</td>\n",
       "      <td>0.628650</td>\n",
       "      <td>0.659189</td>\n",
       "      <td>7211.731060</td>\n",
       "      <td>541.195745</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>0.170402</td>\n",
       "      <td>0.059249</td>\n",
       "      <td>0.182968</td>\n",
       "      <td>0.090652</td>\n",
       "      <td>0.168385</td>\n",
       "      <td>0.058211</td>\n",
       "      <td>0.622101</td>\n",
       "      <td>0.641239</td>\n",
       "      <td>7211.826855</td>\n",
       "      <td>515.451430</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.229221</td>\n",
       "      <td>0.075511</td>\n",
       "      <td>0.236422</td>\n",
       "      <td>0.140594</td>\n",
       "      <td>0.227692</td>\n",
       "      <td>0.074984</td>\n",
       "      <td>0.614612</td>\n",
       "      <td>0.571274</td>\n",
       "      <td>7211.906763</td>\n",
       "      <td>497.050391</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.172103</td>\n",
       "      <td>0.097493</td>\n",
       "      <td>0.191626</td>\n",
       "      <td>0.147746</td>\n",
       "      <td>0.170467</td>\n",
       "      <td>0.096312</td>\n",
       "      <td>0.611279</td>\n",
       "      <td>0.558891</td>\n",
       "      <td>7211.776767</td>\n",
       "      <td>522.665821</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"4\" valign=\"top\">HBOS</th>\n",
       "      <th rowspan=\"4\" valign=\"top\">n_bins</th>\n",
       "      <th>20</th>\n",
       "      <td>0.158367</td>\n",
       "      <td>0.052548</td>\n",
       "      <td>0.208339</td>\n",
       "      <td>0.091463</td>\n",
       "      <td>0.159457</td>\n",
       "      <td>0.046809</td>\n",
       "      <td>0.592251</td>\n",
       "      <td>0.572410</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11.138229</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.137928</td>\n",
       "      <td>0.038511</td>\n",
       "      <td>0.210559</td>\n",
       "      <td>0.100635</td>\n",
       "      <td>0.158408</td>\n",
       "      <td>0.051714</td>\n",
       "      <td>0.580598</td>\n",
       "      <td>0.543764</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.727351</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.129329</td>\n",
       "      <td>0.030647</td>\n",
       "      <td>0.200496</td>\n",
       "      <td>0.085630</td>\n",
       "      <td>0.151286</td>\n",
       "      <td>0.051128</td>\n",
       "      <td>0.569957</td>\n",
       "      <td>0.532903</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10.696349</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.107399</td>\n",
       "      <td>0.026587</td>\n",
       "      <td>0.202059</td>\n",
       "      <td>0.087433</td>\n",
       "      <td>0.150372</td>\n",
       "      <td>0.060479</td>\n",
       "      <td>0.543871</td>\n",
       "      <td>0.511609</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.909404</td>\n",
       "      <td>155</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                AVERAGE_PRECISION            \\\n",
       "                                                             mean    median   \n",
       "algorithm optim_param_name    optim_param_value                               \n",
       "PST       n_bins              5                          0.397080  0.317714   \n",
       "                              8                          0.272842  0.076066   \n",
       "                              10                         0.202958  0.035208   \n",
       "                              20                         0.072902  0.014033   \n",
       "MultiHMM  n_bins              8                          0.039097  0.014123   \n",
       "                              5                          0.069705  0.016897   \n",
       "                              10                         0.031629  0.013809   \n",
       "                              20                         0.031147  0.011715   \n",
       "MTAD-GAT  score_window_size   52                         0.208309  0.094298   \n",
       "                              28                         0.208844  0.091680   \n",
       "                              40                         0.195383  0.074781   \n",
       "          mag_window_size     40                         0.231323  0.096916   \n",
       "                              28                         0.216599  0.094298   \n",
       "                              52                         0.236469  0.097493   \n",
       "          context_window_size 30                         0.260921  0.097392   \n",
       "                              40                         0.198362  0.097154   \n",
       "                              50                         0.170402  0.059249   \n",
       "                              5                          0.229221  0.075511   \n",
       "                              10                         0.172103  0.097493   \n",
       "HBOS      n_bins              20                         0.158367  0.052548   \n",
       "                              10                         0.137928  0.038511   \n",
       "                              8                          0.129329  0.030647   \n",
       "                              5                          0.107399  0.026587   \n",
       "\n",
       "                                                RANGE_PR_AUC            \\\n",
       "                                                        mean    median   \n",
       "algorithm optim_param_name    optim_param_value                          \n",
       "PST       n_bins              5                     0.385879  0.302098   \n",
       "                              8                     0.268959  0.120117   \n",
       "                              10                    0.220957  0.101009   \n",
       "                              20                    0.119969  0.064917   \n",
       "MultiHMM  n_bins              8                     0.133103  0.026042   \n",
       "                              5                     0.130878  0.022867   \n",
       "                              10                    0.143500  0.027247   \n",
       "                              20                    0.156079  0.039948   \n",
       "MTAD-GAT  score_window_size   52                    0.219297  0.137429   \n",
       "                              28                    0.225610  0.151763   \n",
       "                              40                    0.210971  0.117486   \n",
       "          mag_window_size     40                    0.222969  0.110509   \n",
       "                              28                    0.224883  0.143278   \n",
       "                              52                    0.244833  0.168187   \n",
       "          context_window_size 30                    0.254209  0.142827   \n",
       "                              40                    0.214367  0.161455   \n",
       "                              50                    0.182968  0.090652   \n",
       "                              5                     0.236422  0.140594   \n",
       "                              10                    0.191626  0.147746   \n",
       "HBOS      n_bins              20                    0.208339  0.091463   \n",
       "                              10                    0.210559  0.100635   \n",
       "                              8                     0.200496  0.085630   \n",
       "                              5                     0.202059  0.087433   \n",
       "\n",
       "                                                   PR_AUC             ROC_AUC  \\\n",
       "                                                     mean    median      mean   \n",
       "algorithm optim_param_name    optim_param_value                                 \n",
       "PST       n_bins              5                  0.396243  0.317273  0.802249   \n",
       "                              8                  0.271491  0.074917  0.698583   \n",
       "                              10                 0.202411  0.034813  0.592948   \n",
       "                              20                 0.072639  0.013889  0.399095   \n",
       "MultiHMM  n_bins              8                  0.038271  0.009875  0.476832   \n",
       "                              5                  0.070247  0.011805  0.470824   \n",
       "                              10                 0.033808  0.009214  0.461211   \n",
       "                              20                 0.029807  0.009737  0.452648   \n",
       "MTAD-GAT  score_window_size   52                 0.206107  0.092904  0.650048   \n",
       "                              28                 0.207560  0.089998  0.612438   \n",
       "                              40                 0.193867  0.073706  0.606485   \n",
       "          mag_window_size     40                 0.228938  0.096297  0.668495   \n",
       "                              28                 0.214355  0.092904  0.627630   \n",
       "                              52                 0.235039  0.096312  0.616610   \n",
       "          context_window_size 30                 0.259156  0.096297  0.666837   \n",
       "                              40                 0.196483  0.096053  0.628650   \n",
       "                              50                 0.168385  0.058211  0.622101   \n",
       "                              5                  0.227692  0.074984  0.614612   \n",
       "                              10                 0.170467  0.096312  0.611279   \n",
       "HBOS      n_bins              20                 0.159457  0.046809  0.592251   \n",
       "                              10                 0.158408  0.051714  0.580598   \n",
       "                              8                  0.151286  0.051128  0.569957   \n",
       "                              5                  0.150372  0.060479  0.543871   \n",
       "\n",
       "                                                          train_main_time  \\\n",
       "                                                   median            mean   \n",
       "algorithm optim_param_name    optim_param_value                             \n",
       "PST       n_bins              5                  0.894860             NaN   \n",
       "                              8                  0.794044             NaN   \n",
       "                              10                 0.689568             NaN   \n",
       "                              20                 0.298680             NaN   \n",
       "MultiHMM  n_bins              8                  0.471447      363.003773   \n",
       "                              5                  0.480316       97.734562   \n",
       "                              10                 0.460252      342.761122   \n",
       "                              20                 0.467494      427.667099   \n",
       "MTAD-GAT  score_window_size   52                 0.653008     7211.930384   \n",
       "                              28                 0.627799     7211.671927   \n",
       "                              40                 0.579363     7211.709492   \n",
       "          mag_window_size     40                 0.685376     7211.614644   \n",
       "                              28                 0.603581     7211.677360   \n",
       "                              52                 0.571274     7211.752869   \n",
       "          context_window_size 30                 0.698582     7211.686857   \n",
       "                              40                 0.659189     7211.731060   \n",
       "                              50                 0.641239     7211.826855   \n",
       "                              5                  0.571274     7211.906763   \n",
       "                              10                 0.558891     7211.776767   \n",
       "HBOS      n_bins              20                 0.572410             NaN   \n",
       "                              10                 0.543764             NaN   \n",
       "                              8                  0.532903             NaN   \n",
       "                              5                  0.511609             NaN   \n",
       "\n",
       "                                                execute_main_time repetition  \n",
       "                                                             mean      count  \n",
       "algorithm optim_param_name    optim_param_value                               \n",
       "PST       n_bins              5                         34.725921        135  \n",
       "                              8                         35.959607        135  \n",
       "                              10                        37.905403        135  \n",
       "                              20                        46.914917        135  \n",
       "MultiHMM  n_bins              8                          6.446919        155  \n",
       "                              5                          6.673174        155  \n",
       "                              10                         6.536975        155  \n",
       "                              20                         6.671899        155  \n",
       "MTAD-GAT  score_window_size   52                       503.538815        155  \n",
       "                              28                       564.458770        155  \n",
       "                              40                       525.772179        155  \n",
       "          mag_window_size     40                       525.751523        155  \n",
       "                              28                       566.705334        155  \n",
       "                              52                       513.039253        155  \n",
       "          context_window_size 30                       515.276039        155  \n",
       "                              40                       541.195745        155  \n",
       "                              50                       515.451430        155  \n",
       "                              5                        497.050391        155  \n",
       "                              10                       522.665821        155  \n",
       "HBOS      n_bins              20                        11.138229        155  \n",
       "                              10                        10.727351        155  \n",
       "                              8                         10.696349        155  \n",
       "                              5                         15.909404        155  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sort_by = (\"ROC_AUC\", \"mean\")\n",
    "metric_agg_type = [\"mean\", \"median\"]\n",
    "time_agg_type = \"mean\"\n",
    "aggs = {\n",
    "    \"AVERAGE_PRECISION\": metric_agg_type,\n",
    "    \"RANGE_PR_AUC\": metric_agg_type,\n",
    "    \"PR_AUC\": metric_agg_type,\n",
    "    \"ROC_AUC\": metric_agg_type,\n",
    "    \"train_main_time\": time_agg_type,\n",
    "    \"execute_main_time\": time_agg_type,\n",
    "    \"repetition\": \"count\"\n",
    "}\n",
    "\n",
    "df_tmp = df.reset_index()\n",
    "df_tmp = df_tmp.groupby(by=[\"algorithm\", \"optim_param_name\", \"optim_param_value\"]).agg(aggs)\n",
    "df_tmp = df_tmp.reset_index()\n",
    "df_tmp = df_tmp.sort_values(by=[\"algorithm\", \"optim_param_name\", sort_by], ascending=False)\n",
    "df_tmp = df_tmp.set_index([\"algorithm\", \"optim_param_name\", \"optim_param_value\"])\n",
    "\n",
    "with pd.option_context(\"display.max_rows\", None, \"display.max_columns\", None):\n",
    "    display(df_tmp)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7992833e",
   "metadata": {},
   "source": [
    "#### Selected parameters\n",
    "\n",
    "- HBOS: `n_bins=20` (more is better)\n",
    "- MultiHMM: `n_bins=5` (8 is slightly better, but takes way longer. The scores are very bad anyway!)\n",
    "- MTAD-GAT: `context_window_size=30,mag_window_size=40,score_window_size=52` (very slow)\n",
    "- PST: `n_bins=5` (less is better)\n",
    "\n",
    "> **Note**\n",
    ">\n",
    "> MTAD-GAT is very slow! Exclude from further runs!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52cc7556",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_scores([(\"MultiHMM\", \"n_bins\", 5), (\"MultiHMM\", \"n_bins\", 8)], \"sinus-type-mean\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "65eff4b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_scores([(\"MTAD-GAT\", \"context_window_size\", 30), (\"MTAD-GAT\", \"context_window_size\", 40)], \"sinus-type-mean\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
